Index
#!pip install scikit-fda
import os
os.chdir("..")
# Import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import altair as alt
import random
import statsmodels.api as sm
from skfda.representation.grid import FDataGrid
from skfda.preprocessing.dim_reduction.projection import FPCA
from skfda.exploratory.visualization import FPCAPlot
from sklearn.preprocessing import OneHotEncoder
import skfda
from skfda.ml.regression import LinearRegression
from skfda.representation.basis import FDataBasis, FourierBasis
from skfda.exploratory.depth import IntegratedDepth, ModifiedBandDepth
from skfda.exploratory.visualization import Boxplot
# Import designed-functions
from window_extraction import calculate_window_values, calculate_window_data, Merge_data, align_to_zero, balance_index
from time_series_visualization import plot_all_time_series, plot_all_time_series_and_mean_fpca, plot_all_time_series_in_group
from functionalPCA import fpca_two_inputs, first_component_extraction, bootstrap, create_pc_scores_plots, visualize_regression
from functional_regression import Function_regression, coefficent_visualization
C:\Users\Administrator\AppData\Local\Temp\ipykernel_21156\3302358765.py:9: DeprecationWarning: The module "projection" is deprecated. Please use "dim_reduction" from skfda.preprocessing.dim_reduction.projection import FPCA
The path of the files can be change based on where the data is stored.
# Import datasets
sensorA_System1 = pd.read_csv("RawData/System1_SensorA.csv")
sensorA_System2 = pd.read_csv("RawData/System2_SensorA.csv")
sensorB_System1 = pd.read_csv("RawData/System1_SensorB.csv")
sensorB_System2 = pd.read_csv("RawData/System2_SensorB.csv")
sensorA_System1_missing = pd.read_csv("RawData/SensorA_System1_missing values.csv")
sensorA_System2_missing = pd.read_csv("RawData/SensorA_System2_missing values.csv")
keyByTestID = pd.read_csv("RawData/Key by TestID.csv", parse_dates=['DateTime'])
# Transpose dataset to make columns as timestamps and rows as tests
# Sensor A
A1_transposed = sensorA_System1.T.reset_index()
A1_transposed.columns = A1_transposed.iloc[0]
A1_transposed.rename(columns={A1_transposed.columns[0]: 'TestID'}, inplace=True)
A1_transposed = A1_transposed.drop(0)
A1_transposed['TestID'] = A1_transposed['TestID'].astype(int)
A2_transposed = sensorA_System2.T.reset_index()
A2_transposed.columns = A2_transposed.iloc[0]
A2_transposed.rename(columns={A2_transposed.columns[0]: 'TestID'}, inplace=True)
A2_transposed = A2_transposed.drop(0)
A2_transposed['TestID'] = A2_transposed['TestID'].astype(int)
A1_missing_transposed = sensorA_System1_missing.T.reset_index()
A1_missing_transposed.columns = A1_missing_transposed.iloc[0]
A1_missing_transposed.rename(columns={A1_missing_transposed.columns[0]: 'TestID'}, inplace=True)
A1_missing_transposed = A1_missing_transposed.drop(0)
A1_missing_transposed['TestID'] = A1_missing_transposed['TestID'].astype(int)
A2_missing_transposed = sensorA_System2_missing.T.reset_index()
A2_missing_transposed.columns = A2_missing_transposed.iloc[0]
A2_missing_transposed.rename(columns={A2_missing_transposed.columns[0]: 'TestID'}, inplace=True)
A2_missing_transposed = A2_missing_transposed.drop(0)
A2_missing_transposed['TestID'] = A2_missing_transposed['TestID'].astype(int)
# Sensor B
B1_transposed = sensorB_System1.T.reset_index()
B1_transposed.columns = B1_transposed.iloc[0]
B1_transposed.rename(columns={B1_transposed.columns[0]: 'TestID'}, inplace=True)
B1_transposed = B1_transposed.drop(0)
B1_transposed['TestID'] = B1_transposed['TestID'].astype(int)
B2_transposed = sensorB_System2.T.reset_index()
B2_transposed.columns = B2_transposed.iloc[0]
B2_transposed.rename(columns={B2_transposed.columns[0]: 'TestID'}, inplace=True)
B2_transposed = B2_transposed.drop(0)
B2_transposed['TestID'] = B2_transposed['TestID'].astype(int)
# Complete A1 and A2 with the missing values
A1_transposed_mid = A1_transposed[~A1_transposed.TestID.isin(A1_missing_transposed.TestID)]
A1_transposed = pd.concat([A1_transposed_mid, A1_missing_transposed], axis=0)
A2_transposed_mid = A2_transposed[~A2_transposed.TestID.isin(A2_missing_transposed.TestID)]
A2_transposed = pd.concat([A2_transposed_mid, A2_missing_transposed], axis=0)
# Relabeling System Values
keyByTestID["System"] = keyByTestID["System"].replace({"System 2A":"System 2","System 2B":"System 2"})
# Create new column to fill fluid temperature NA's
# Note: Fluid temperature: If specified, take as the temperature of the sample fluid. The rest of the system temperature can be taken as ambient temperature.
keyByTestID['Fluid_Temperature_Filled'] = keyByTestID['Fluid Temperature'].combine_first(keyByTestID['AmbientTemperature'])
# Binning
# Categorize 'FluidType' into Blood and Aqueous
keyByTestID['FluidTypeBin'] = np.where(keyByTestID['FluidType'].str.startswith('Eurotrol'), 'Aqueous', 'Blood')
# Categorize 'AgeOfCardInDaysAtTimeOfTest' into bins
keyByTestID["CardAgeBin"] = pd.cut(keyByTestID["AgeOfCardInDaysAtTimeOfTest"], bins=[0, 9, 28, 56, 84, 112, 140, 168, 196, 224, 252],
labels=['[0-9]', '(9-28]', '(28-56]', '(56-84]', '(84-112]', '(112-140]', '(140-168]', '(168-196]', '(196-224]', '(224-252]'])
# Categorize 'Fluid_Temperature_Filled' into bins
keyByTestID["FluidTempBin"] = pd.cut(keyByTestID["Fluid_Temperature_Filled"], bins=[-1, 20, 25, 100], labels=['Below 20', '20-25', 'Above 25'])
# Filtering successful tests
keyByTestID = keyByTestID[keyByTestID['ReturnCode'].isin(['Success','UnderReportableRange'])]
# Merge dataset with keyByTestID and delete unmatched tests
keyByTestID['TestID'] = keyByTestID['TestID'].astype(int)
keyByTestID['System'] = keyByTestID['System'].astype(str)
A1_keyByTestID = keyByTestID[(keyByTestID['Sensor'] == 'Sensor A') & (keyByTestID['System'] == 'System 1')]
A1_Merged = pd.merge(A1_keyByTestID,A1_transposed,how='inner', on=['TestID'])
A1_transposed = A1_transposed[A1_transposed['TestID'].isin(A1_Merged['TestID'])]
A2_keyByTestID = keyByTestID.loc[(keyByTestID['Sensor'] == 'Sensor A') & (keyByTestID['System'] != 'System 1')]
A2_Merged = pd.merge(A2_keyByTestID,A2_transposed,how='inner', on=['TestID'])
A2_transposed = A2_transposed[A2_transposed['TestID'].isin(A2_Merged['TestID'])]
sensorA_System1 = sensorA_System1.loc[:, sensorA_System1.columns.isin(A1_Merged['TestID'].astype(str))]
sensorA_System2 = sensorA_System2.loc[:, sensorA_System2.columns.isin(A2_Merged['TestID'].astype(str))]
B1_keyByTestID = keyByTestID[(keyByTestID['Sensor'] == 'Sensor B') & (keyByTestID['System'] == 'System 1')]
B1_Merged = pd.merge(B1_keyByTestID,B1_transposed,how='inner', on=['TestID'])
B1_transposed = B1_transposed[B1_transposed['TestID'].isin(B1_Merged['TestID'])]
B2_keyByTestID = keyByTestID.loc[(keyByTestID['Sensor'] == 'Sensor B') & (keyByTestID['System'] != 'System 1')]
B2_Merged = pd.merge(B2_keyByTestID,B2_transposed,how='inner', on=['TestID'])
B1_transposed = B2_transposed[B2_transposed['TestID'].isin(A2_Merged['TestID'])]
sensorB_System1 = sensorB_System1.loc[:, sensorB_System1.columns.isin(B1_Merged['TestID'].astype(str))]
sensorB_System2 = sensorB_System2.loc[:, sensorB_System2.columns.isin(B2_Merged['TestID'].astype(str))]
print('A1: ', A1_Merged.shape)
print('A2: ', A2_Merged.shape)
print('B1: ', B1_Merged.shape)
print('B2: ', B2_Merged.shape)
A1: (3382, 3380) A2: (7743, 3371) B1: (3375, 3380) B2: (7745, 3371)
# Note: Only run once. If not, restart the kernel and run from the beggining again.
A1_Merged = A1_Merged[A1_Merged["TestID"].isin(B1_Merged["TestID"])]
B1_Merged = B1_Merged[B1_Merged["TestID"].isin(A1_Merged["TestID"])]
A2_Merged = A2_Merged[A2_Merged["TestID"].isin(B2_Merged["TestID"])]
B2_Merged = B2_Merged[B2_Merged["TestID"].isin(A2_Merged["TestID"])]
print('A1: ', A1_Merged.shape)
print('A2: ', A2_Merged.shape)
print('B1: ', B1_Merged.shape)
print('B2: ', B2_Merged.shape)
A1: (3374, 3380) A2: (7743, 3371) B1: (3374, 3380) B2: (7743, 3371)
# Match window values of Sensor A for each test
calDelimit = 11
cal_window_size = 8
sampleDelimit = 15
sample_window_size = 5
# Sensor A
cal_window_start, cal_window_end, sample_window_start, sample_window_end = calculate_window_values(bubble_start=A1_Merged['BubbleDetectTime'],
sample_start=A1_Merged['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit,
sample_window_size_input=sample_window_size)
A1_Merged['cal_window_start']=cal_window_start
A1_Merged['cal_window_end']=cal_window_end
A1_Merged['sample_window_start']=sample_window_start
A1_Merged['sample_window_end']=sample_window_end
cal_window_start, cal_window_end, sample_window_start, sample_window_end = calculate_window_values(bubble_start=A2_Merged['BubbleDetectTime'],
sample_start=A2_Merged['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit,
sample_window_size_input=sample_window_size)
A2_Merged['cal_window_start']=cal_window_start
A2_Merged['cal_window_end']=cal_window_end
A2_Merged['sample_window_start']=sample_window_start
A2_Merged['sample_window_end']=sample_window_end
# sensor B
# Match window values of Sensor B for each test
calDelimit = 20
cal_window_size = 18
sampleDelimit_blood = 24
sampleDelimit_aqueous = 30
sample_window_size = 4
B1_Merged['cal_window_start'], B1_Merged['cal_window_end'], \
B1_Merged['sample_window_start'], B1_Merged['sample_window_end'] = zip(*B1_Merged.apply(
lambda row: calculate_window_values(
bubble_start=row['BubbleDetectTime'],
sample_start=row['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit_aqueous if row['FluidType'].startswith('Eurotrol') else sampleDelimit_blood,
sample_window_size_input=sample_window_size
),
axis=1
))
# For sensor B in system 2, blood and aqueous
B2_Merged['cal_window_start'], B2_Merged['cal_window_end'], \
B2_Merged['sample_window_start'], B2_Merged['sample_window_end'] = zip(*B2_Merged.apply(
lambda row: calculate_window_values(
bubble_start=row['BubbleDetectTime'],
sample_start=row['SampleDetectTime'],
calDelimit_input=calDelimit,
cal_window_size_input=cal_window_size,
sampleDelimit_input=sampleDelimit_aqueous if row['FluidType'].startswith('Eurotrol') else sampleDelimit_blood,
sample_window_size_input=sample_window_size
),
axis=1
))
# Adds TestIDs as index to the values post-window extraction
# System 1 - Sensor A
A1_cal_window = []
A1_sample_window = []
for i in range(len(A1_Merged)):
cal_window, sample_window = calculate_window_data(A1_Merged.iloc[i, :])
A1_cal_window.append(cal_window.values)
A1_sample_window.append(sample_window.values)
A1_cal_window = pd.DataFrame(A1_cal_window)
A1_sample_window = pd.DataFrame(A1_sample_window)
A1_cal_window['TestID'] = A1_sample_window['TestID'] = A1_Merged['TestID'].astype(int)
A1_sample_window.set_index('TestID',inplace=True)
A1_cal_window.set_index('TestID',inplace=True)
# System 2 - Sensor A
A2_cal_window = []
A2_sample_window = []
for i in range(len(A2_Merged)):
cal_window, sample_window = calculate_window_data(A2_Merged.iloc[i, :])
A2_cal_window.append(cal_window.values)
A2_sample_window.append(sample_window.values)
A2_cal_window = pd.DataFrame(A2_cal_window)
A2_sample_window = pd.DataFrame(A2_sample_window)
A2_cal_window['TestID'] = A2_sample_window['TestID'] = A2_Merged['TestID'].astype(int)
A2_sample_window.set_index('TestID',inplace=True)
A2_cal_window.set_index('TestID',inplace=True)
# System 1 - Sensor B
B1_cal_window = []
B1_sample_window = []
for i in range(len(B1_Merged)):
cal_window, sample_window = calculate_window_data(B1_Merged.iloc[i, :])
B1_cal_window.append(cal_window.values)
B1_sample_window.append(sample_window.values)
B1_cal_window = pd.DataFrame(B1_cal_window)
B1_sample_window = pd.DataFrame(B1_sample_window)
B1_cal_window['TestID'] = B1_sample_window['TestID'] = B1_Merged['TestID'].astype(int)
B1_sample_window.set_index('TestID',inplace=True)
B1_cal_window.set_index('TestID',inplace=True)
# System 2 - Sensor B
B2_cal_window = []
B2_sample_window = []
for i in range(len(B2_Merged)):
cal_window, sample_window = calculate_window_data(B2_Merged.iloc[i, :])
B2_cal_window.append(cal_window.values)
B2_sample_window.append(sample_window.values)
B2_cal_window = pd.DataFrame(B2_cal_window)
B2_sample_window = pd.DataFrame(B2_sample_window)
B2_cal_window['TestID'] = B2_sample_window['TestID'] = B2_Merged['TestID'].astype(int)
B2_sample_window.set_index('TestID',inplace=True)
B2_cal_window.set_index('TestID',inplace=True)
A1_cal_window_drop_index = A1_cal_window.loc[A1_cal_window.isna().sum(axis=1)!=0].index
A2_cal_window_drop_index = A2_cal_window.loc[A2_cal_window.isna().sum(axis=1)!=0].index
A1_sample_window_drop_index = A1_sample_window.loc[A1_sample_window.isna().sum(axis=1)!=0].index
A2_sample_window_drop_index = A2_sample_window.loc[A2_sample_window.isna().sum(axis=1)!=0].index
B1_cal_window_drop_index = B1_cal_window.loc[B1_cal_window.isna().sum(axis=1)!=0].index
B2_cal_window_drop_index = B2_cal_window.loc[B2_cal_window.isna().sum(axis=1)!=0].index
B1_sample_window_drop_index = B1_sample_window.loc[B1_sample_window.isna().sum(axis=1)!=0].index
B2_sample_window_drop_index = B2_sample_window.loc[B2_sample_window.isna().sum(axis=1)!=0].index
# Check if missing values in different windows is different
print("The missing value in calibration window:",A1_cal_window_drop_index)
print("The missing value in sample window:",A1_sample_window_drop_index)
print("The missing value in calibration window:",A2_cal_window_drop_index)
print("The missing value in sample window:",A2_sample_window_drop_index)
print("The missing value in calibration window:",B1_cal_window_drop_index)
print("The missing value in sample window:",B1_sample_window_drop_index)
print("The missing value in calibration window:",B2_cal_window_drop_index)
print("The missing value in sample window:",B2_sample_window_drop_index)
The missing value in calibration window: Float64Index([], dtype='float64', name='TestID') The missing value in sample window: Float64Index([], dtype='float64', name='TestID') The missing value in calibration window: Int64Index([], dtype='int64', name='TestID') The missing value in sample window: Int64Index([], dtype='int64', name='TestID') The missing value in calibration window: Float64Index([], dtype='float64', name='TestID') The missing value in sample window: Float64Index([], dtype='float64', name='TestID') The missing value in calibration window: Float64Index([], dtype='float64', name='TestID') The missing value in sample window: Float64Index([], dtype='float64', name='TestID')
# Set index for Merge datasets
A1_Merged.set_index("TestID", inplace=True)
A2_Merged.set_index("TestID", inplace=True)
B1_Merged.set_index("TestID", inplace=True)
B2_Merged.set_index("TestID", inplace=True)
# Find missing value
print("The problem indexes after extract the window are:",A1_Merged.index.difference(A1_cal_window.index))
print("The problem indexes after extract the window are:",A1_Merged.index.difference(A1_sample_window.index))
print("The problem indexes after extract the window are:",A2_Merged.index.difference(A2_cal_window.index))
print("The problem indexes after extract the window are:",A2_Merged.index.difference(A2_sample_window.index))
print("The problem indexes after extract the window are:",B1_Merged.index.difference(B1_cal_window.index))
print("The problem indexes after extract the window are:",B1_Merged.index.difference(B1_sample_window.index))
print("The problem indexes after extract the window are:",B2_Merged.index.difference(B2_cal_window.index))
print("The problem indexes after extract the window are:",B2_Merged.index.difference(B2_sample_window.index))
A1_Merged = A1_Merged.drop(A1_Merged.index.difference(A1_cal_window.index))
A1_Merged = A1_Merged.drop(A1_Merged.index.difference(A1_sample_window.index))
A2_Merged = A2_Merged.drop(A2_Merged.index.difference(A2_cal_window.index))
A2_Merged = A2_Merged.drop(A2_Merged.index.difference(A2_sample_window.index))
B1_Merged = B1_Merged.drop(B1_Merged.index.difference(B1_cal_window.index))
B1_Merged = B1_Merged.drop(B1_Merged.index.difference(B1_sample_window.index))
B2_Merged = B2_Merged.drop(B2_Merged.index.difference(B2_cal_window.index))
B2_Merged = B2_Merged.drop(B2_Merged.index.difference(B2_sample_window.index))
# Clear the Nan in index of sensor A
A1_cal_window = A1_cal_window[~A1_cal_window.index.isna()]
A1_sample_window = A1_sample_window[~A1_sample_window.index.isna()]
A2_cal_window = A2_cal_window[~A2_cal_window.index.isna()]
A2_sample_window = A2_sample_window[~A2_sample_window.index.isna()]
# Clear the Nan in index of sensor B
B1_cal_window = B1_cal_window[~B1_cal_window.index.isna()]
B1_sample_window = B1_sample_window[~B1_sample_window.index.isna()]
B2_cal_window = B2_cal_window[~B2_cal_window.index.isna()]
B2_sample_window = B2_sample_window[~B2_sample_window.index.isna()]
The problem indexes after extract the window are: Int64Index([12470355, 12470361, 12470365, 12537663, 12539049, 12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([12470355, 12470361, 12470365, 12537663, 12539049, 12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([12622570], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([3518677, 3518678], dtype='int64', name='TestID') The problem indexes after extract the window are: Int64Index([3518677, 3518678], dtype='int64', name='TestID')
# Shape of the subsets of time series after the extraction from the windows
# Cal Window
print('Shape of the time series after extraction')
print('A1_cal_window: ', A1_cal_window.shape)
print('A2_cal_window: ', A2_cal_window.shape)
print('B1_cal_window: ', B1_cal_window.shape)
print('B2_cal_window: ', B2_cal_window.shape)
# Sample Window
print('A1_sample_window: ', A1_sample_window.shape)
print('A2_sample_window: ', A2_sample_window.shape)
print('B1_sample_window: ', B1_sample_window.shape)
print('B2_sample_window: ', B2_sample_window.shape)
# We can delete the unmatch index but it is not necessary
Shape of the time series after extraction A1_cal_window: (3368, 41) A2_cal_window: (7743, 41) B1_cal_window: (3373, 91) B2_cal_window: (7741, 91) A1_sample_window: (3368, 26) A2_sample_window: (7743, 26) B1_sample_window: (3373, 21) B2_sample_window: (7741, 21)
# Cal Window
A1_cal_window_zero = align_to_zero(A1_cal_window)
A2_cal_window_zero = align_to_zero(A2_cal_window)
B1_cal_window_zero = align_to_zero(B1_cal_window)
B2_cal_window_zero = align_to_zero(B2_cal_window)
# Sample Window
A1_sample_window_zero = align_to_zero(A1_sample_window)
A2_sample_window_zero = align_to_zero(A2_sample_window)
B1_sample_window_zero = align_to_zero(B1_sample_window)
B2_sample_window_zero = align_to_zero(B2_sample_window)
# Combine data: Merge the zero-aligned time series with "FluidType", "AgeOfCardInDaysAtTimeOfTest", "Fluid_Temperature_Filled", "FluidTypeBin", "CardAgeBin", "FluidTempBin"
A1_cal_window_combine = Merge_data(A1_cal_window_zero,A1_Merged)
A2_cal_window_combine = Merge_data(A2_cal_window_zero,A2_Merged)
B1_cal_window_combine = Merge_data(B1_cal_window_zero,B1_Merged)
B2_cal_window_combine = Merge_data(B2_cal_window_zero,B2_Merged)
## Sample window
A1_sample_window_combine = Merge_data(A1_sample_window_zero,A1_Merged)
A2_sample_window_combine = Merge_data(A2_sample_window_zero,A2_Merged)
B1_sample_window_combine = Merge_data(B1_sample_window_zero,B1_Merged)
B2_sample_window_combine = Merge_data(B2_sample_window_zero,B2_Merged)
System1_Index, System2_Index = balance_index(A1_cal_window_combine,A2_cal_window_combine,"CardAgeBin")
System1 Sensor A & B distribution: [0-9] 142 (9-28] 142 (28-56] 142 (56-84] 142 (84-112] 142 (112-140] 142 (140-168] 142 (168-196] 142 (196-224] 142 (224-252] 142 Name: CardAgeBin, dtype: int64 System2 Sensor A & B distribution: [0-9] 142 (9-28] 142 (28-56] 142 (56-84] 142 (84-112] 142 (112-140] 142 (140-168] 142 (168-196] 142 (196-224] 142 (224-252] 142 Name: CardAgeBin, dtype: int64
# Balanced data
A1_cal_window_combine_balanced = A1_cal_window_combine.loc[System1_Index]
A1_sample_window_combine_balanced = A1_sample_window_combine.loc[System1_Index]
A2_cal_window_combine_balanced = A2_cal_window_combine.loc[System2_Index]
A2_sample_window_combine_balanced = A2_sample_window_combine.loc[System2_Index]
B1_cal_window_combine_balanced = B1_cal_window_combine.loc[System1_Index]
B1_sample_window_combine_balanced = B1_sample_window_combine.loc[System1_Index]
B2_cal_window_combine_balanced = B2_cal_window_combine.loc[System2_Index]
B2_sample_window_combine_balanced = B2_sample_window_combine.loc[System2_Index]
# Plot all the balanced time series from the window extraction
plot_all_time_series_in_group(A1_cal_window_combine_balanced, A1_sample_window_combine_balanced, A2_cal_window_combine_balanced, A2_sample_window_combine_balanced, "CardAgeBin", "A1_cal_window_combine", "A1_sample_window_combine","A2_blood_cal_window_combine", "A2_sample_window_combine")
# Plot all the balanced time series from the window extraction
plot_all_time_series_in_group(B1_cal_window_combine_balanced, B1_sample_window_combine_balanced, B2_cal_window_combine_balanced, B2_sample_window_combine_balanced, "CardAgeBin", "B1_cal_window_combine", "B1_sample_window_combine", "B2_blood_cal_window_combine", "B2_sample_window_combine")
The following secssion will introduce
pc_scores_s1_A_cal_window, pc_scores_s2_A_cal_window,fpca_s1_A_cal_window,fpca_s2_A_cal_window = fpca_two_inputs(A1_cal_window_combine_balanced.iloc[:,:-6], A2_cal_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
ac1, ac2 = bootstrap(A1_cal_window_combine_balanced, A2_cal_window_combine_balanced,"A","cal_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_A_cal_window, pc_scores_s2_A_cal_window, A1_cal_window_combine_balanced, A2_cal_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.87177417445558 S1 Explain variance PC2 (%): 0.031779536988427504 S2 Explain variance PC1 (%): 99.93899476648986 S2 Explain variance PC2 (%): 0.02271454773551496 The time series contributing most to PC1 is at index 800 with TestID 12529762.0 The time series contributing most to PC2 is at index 82 with TestID 12615989.0 The time series contributing most to PC1 is at index 91 with TestID 3568638 The time series contributing most to PC2 is at index 19 with TestID 3559978
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
pc_scores_s1_A_sample_window, pc_scores_s2_A_sample_window,fpca_s1_A_sample_window,fpca_s2_A_sample_window = fpca_two_inputs(A1_sample_window_combine_balanced.iloc[:,:-6], A2_sample_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
as1,as2 = bootstrap(A1_sample_window_combine_balanced, A2_sample_window_combine_balanced,"A","sample_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_A_sample_window, pc_scores_s2_A_sample_window, A1_sample_window_combine_balanced, A2_sample_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.54001643310654 S1 Explain variance PC2 (%): 0.13376186892582478 S2 Explain variance PC1 (%): 99.83602096130886 S2 Explain variance PC2 (%): 0.06238709532612155 The time series contributing most to PC1 is at index 800 with TestID 12529762.0 The time series contributing most to PC2 is at index 261 with TestID 12515884.0 The time series contributing most to PC1 is at index 140 with TestID 3568703 The time series contributing most to PC2 is at index 742 with TestID 3555912
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
pc_scores_s1_B_cal_window, pc_scores_s2_B_cal_window,fpca_s1_B_cal_window,fpca_s2_B_cal_window = fpca_two_inputs(B1_cal_window_combine_balanced.iloc[:,:-6], B2_cal_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
bc1,bc2 = bootstrap(B1_cal_window_combine_balanced, B2_cal_window_combine_balanced,"B","cal_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_B_cal_window, pc_scores_s2_B_cal_window, B1_cal_window_combine_balanced, B2_cal_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.85049627222725 S1 Explain variance PC2 (%): 0.08940797879890665 S2 Explain variance PC1 (%): 99.87297367499667 S2 Explain variance PC2 (%): 0.09672229658461992 The time series contributing most to PC1 is at index 82 with TestID 12615989.0 The time series contributing most to PC2 is at index 664 with TestID 12371094.0 The time series contributing most to PC1 is at index 53 with TestID 3565690.0 The time series contributing most to PC2 is at index 53 with TestID 3565690.0
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
pc_scores_s1_B_sample_window, pc_scores_s2_B_sample_window,fpca_s1_B_sample_window,fpca_s2_B_sample_window = fpca_two_inputs(B1_sample_window_combine_balanced.iloc[:,:-6], B2_sample_window_combine_balanced.iloc[:,:-6], color_fpc1_s1='tab:blue', color_fpc2_s1='tab:cyan', color_fpc1_s2='tab:orange', color_fpc2_s2='gold')
print("--------------------------------------------------- Bootstrap -------------------------------------------------------------------------------------------")
bs1,bs2 = bootstrap(B1_sample_window_combine_balanced, B2_sample_window_combine_balanced, "B","sample_window",features="CardAgeBin")
print("--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------")
create_pc_scores_plots(pc_scores_s1_B_sample_window, pc_scores_s2_B_sample_window, B1_sample_window_combine_balanced, B2_sample_window_combine_balanced,features="CardAgeBin")
S1 Explain variance PC1 (%): 99.79207638417347 S1 Explain variance PC2 (%): 0.058415725244599884 S2 Explain variance PC1 (%): 99.88973695475921 S2 Explain variance PC2 (%): 0.04589281414980153 The time series contributing most to PC1 is at index 684 with TestID 12191141.0 The time series contributing most to PC2 is at index 103 with TestID 12581955.0 The time series contributing most to PC1 is at index 666 with TestID 3518710.0 The time series contributing most to PC2 is at index 120 with TestID 3566587.0
--------------------------------------------------- Bootstrap ------------------------------------------------------------------------------------------- Confidence Interval of 1st component The number of sampling is 142 The boxplot of 1st Component
--------------------------------------------------- PCA Scores -------------------------------------------------------------------------------------------
df_list = []
def append_to_dataframe(window_name, slope1, slope2):
global df_list
df_list.append({'Window': window_name, 'Slope 1': slope1, 'Slope 2': slope2})
append_to_dataframe('A_cal_window', *visualize_regression(fpca_s1_A_cal_window, fpca_s2_A_cal_window))
append_to_dataframe('A_sample_window', *visualize_regression(fpca_s1_A_sample_window, fpca_s2_A_sample_window))
append_to_dataframe('B_cal_window', *visualize_regression(fpca_s1_B_cal_window, fpca_s2_B_cal_window))
append_to_dataframe('B_sample_window', *visualize_regression(fpca_s1_B_sample_window, fpca_s2_B_sample_window))
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 7.931e+05
Date: Wed, 12 Jun 2024 Prob (F-statistic): 1.09e-83
Time: 21:05:34 Log-Likelihood: 242.45
No. Observations: 40 AIC: -480.9
Df Residuals: 38 BIC: -477.5
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0058 0.000 30.977 0.000 0.005 0.006
x1 -0.0071 7.93e-06 -890.548 0.000 -0.007 -0.007
==============================================================================
Omnibus: 3.406 Durbin-Watson: 0.109
Prob(Omnibus): 0.182 Jarque-Bera (JB): 3.119
Skew: 0.618 Prob(JB): 0.210
Kurtosis: 2.414 Cond. No. 48.0
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 8.805e+05
Date: Wed, 12 Jun 2024 Prob (F-statistic): 1.49e-84
Time: 21:05:34 Log-Likelihood: 244.49
No. Observations: 40 AIC: -485.0
Df Residuals: 38 BIC: -481.6
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0060 0.000 33.868 0.000 0.006 0.006
x1 -0.0071 7.53e-06 -938.366 0.000 -0.007 -0.007
==============================================================================
Omnibus: 5.737 Durbin-Watson: 0.083
Prob(Omnibus): 0.057 Jarque-Bera (JB): 3.129
Skew: 0.462 Prob(JB): 0.209
Kurtosis: 1.988 Cond. No. 48.0
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 5.446e+05
Date: Wed, 12 Jun 2024 Prob (F-statistic): 8.16e-52
Time: 21:05:35 Log-Likelihood: 146.38
No. Observations: 25 AIC: -288.8
Df Residuals: 23 BIC: -286.3
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0157 0.000 52.697 0.000 0.015 0.016
x1 -0.0148 2e-05 -737.942 0.000 -0.015 -0.015
==============================================================================
Omnibus: 0.529 Durbin-Watson: 0.661
Prob(Omnibus): 0.768 Jarque-Bera (JB): 0.461
Skew: -0.296 Prob(JB): 0.794
Kurtosis: 2.698 Cond. No. 30.8
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 7.501e+05
Date: Wed, 12 Jun 2024 Prob (F-statistic): 2.05e-53
Time: 21:05:35 Log-Likelihood: 150.62
No. Observations: 25 AIC: -297.2
Df Residuals: 23 BIC: -294.8
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0134 0.000 53.364 0.000 0.013 0.014
x1 -0.0147 1.69e-05 -866.059 0.000 -0.015 -0.015
==============================================================================
Omnibus: 3.255 Durbin-Watson: 0.195
Prob(Omnibus): 0.196 Jarque-Bera (JB): 2.209
Skew: 0.543 Prob(JB): 0.331
Kurtosis: 2.029 Cond. No. 30.8
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 1.000
Model: OLS Adj. R-squared: 1.000
Method: Least Squares F-statistic: 2.793e+05
Date: Wed, 12 Jun 2024 Prob (F-statistic): 7.17e-156
Time: 21:05:36 Log-Likelihood: 499.94
No. Observations: 90 AIC: -995.9
Df Residuals: 88 BIC: -990.9
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const -0.0001 0.000 -0.508 0.613 -0.001 0.000
x1 0.0020 3.84e-06 528.450 0.000 0.002 0.002
==============================================================================
Omnibus: 13.949 Durbin-Watson: 0.016
Prob(Omnibus): 0.001 Jarque-Bera (JB): 8.997
Skew: -0.629 Prob(JB): 0.0111
Kurtosis: 2.097 Cond. No. 106.
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.999
Model: OLS Adj. R-squared: 0.999
Method: Least Squares F-statistic: 1.663e+05
Date: Wed, 12 Jun 2024 Prob (F-statistic): 5.66e-146
Time: 21:05:36 Log-Likelihood: 477.27
No. Observations: 90 AIC: -950.5
Df Residuals: 88 BIC: -945.5
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0008 0.000 2.973 0.004 0.000 0.001
x1 0.0020 4.94e-06 407.834 0.000 0.002 0.002
==============================================================================
Omnibus: 11.542 Durbin-Watson: 0.009
Prob(Omnibus): 0.003 Jarque-Bera (JB): 9.163
Skew: -0.675 Prob(JB): 0.0102
Kurtosis: 2.211 Cond. No. 106.
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.999
Model: OLS Adj. R-squared: 0.999
Method: Least Squares F-statistic: 1.747e+04
Date: Wed, 12 Jun 2024 Prob (F-statistic): 2.40e-28
Time: 21:05:36 Log-Likelihood: 83.320
No. Observations: 20 AIC: -162.6
Df Residuals: 18 BIC: -160.6
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0128 0.002 6.937 0.000 0.009 0.017
x1 -0.0203 0.000 -132.180 0.000 -0.021 -0.020
==============================================================================
Omnibus: 2.319 Durbin-Watson: 0.138
Prob(Omnibus): 0.314 Jarque-Bera (JB): 1.832
Skew: 0.609 Prob(JB): 0.400
Kurtosis: 2.154 Cond. No. 25.0
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
OLS Regression Results
==============================================================================
Dep. Variable: y R-squared: 0.999
Model: OLS Adj. R-squared: 0.999
Method: Least Squares F-statistic: 1.861e+04
Date: Wed, 12 Jun 2024 Prob (F-statistic): 1.36e-28
Time: 21:05:36 Log-Likelihood: 83.924
No. Observations: 20 AIC: -163.8
Df Residuals: 18 BIC: -161.9
Df Model: 1
Covariance Type: nonrobust
==============================================================================
coef std err t P>|t| [0.025 0.975]
------------------------------------------------------------------------------
const 0.0131 0.002 7.367 0.000 0.009 0.017
x1 -0.0203 0.000 -136.433 0.000 -0.021 -0.020
==============================================================================
Omnibus: 2.598 Durbin-Watson: 0.133
Prob(Omnibus): 0.273 Jarque-Bera (JB): 1.958
Skew: 0.617 Prob(JB): 0.376
Kurtosis: 2.092 Cond. No. 25.0
==============================================================================
Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
slopes_df = pd.DataFrame(df_list)
slopes_df
| Window | Slope 1 | Slope 2 | |
|---|---|---|---|
| 0 | A_cal_window | -0.007061 | -0.007069 |
| 1 | A_sample_window | -0.014793 | -0.014652 |
| 2 | B_cal_window | 0.002030 | 0.002015 |
| 3 | B_sample_window | -0.020283 | -0.020312 |
This is another functional analysis method. Unlike FPCA, the following analysis utilizes the entire time series in a balanced and centered dataset as response variables for regression with the features before grouping by bins. This is done to distinguish between two systems under the influence of features.
This is the coeffcient from the output of the model. Because of the different magnitude, we need to choose the time stamps before we visualize
print("System 1:")
A1_cal_window_funct_reg = Function_regression(A1_cal_window_combine_balanced,40,['AgeOfCardInDaysAtTimeOfTest'])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
A2_cal_window_funct_reg = Function_regression(A2_cal_window_combine_balanced,40,['AgeOfCardInDaysAtTimeOfTest'])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 39.0),), n_basis=41, period=39.0),
coefficients=[[ 5.50128277e-01 -2.70867850e-01 -2.73647747e-02 -1.02878155e-01
-2.87858877e-02 -6.06709294e-02 -2.87103246e-03 -8.12933499e-02
1.46661206e-02 -6.19589794e-02 -9.92147448e-05 -3.72463402e-02
5.03205595e-04 -3.30376122e-02 -3.88200892e-02 8.03740547e-03
5.05561188e-03 -2.25536036e-02 -2.62058044e-02 -2.23841757e-02
3.21238254e-03 -4.76579594e-02 8.28678180e-05 -1.84643694e-03
-2.50885594e-02 -5.16148864e-02 -1.77625277e-02 -3.67125797e-02
-4.11417891e-03 -2.49144092e-02 -1.17534243e-02 -9.16334419e-02
1.65788378e-02 -8.93649865e-02 1.53017251e-02 -1.78821968e-01
-8.66927544e-03 -2.53570098e+13 1.65349010e+11 -2.53570098e+13
-1.65349010e+11]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 39.0),), n_basis=41, period=39.0),
coefficients=[[ 1.18903452e-02 -5.31779019e-03 -7.45129153e-04 -2.03577242e-03
-6.94473934e-04 -1.24164555e-03 1.31317135e-04 -9.07379050e-04
2.12847105e-04 -1.15455221e-03 1.54440916e-04 -1.12929847e-03
-1.83123230e-04 -9.08998342e-04 3.94723653e-05 -8.71028251e-04
-1.30652939e-04 -7.37835637e-04 3.67765375e-04 -1.14774705e-03
2.25360980e-04 -9.46250927e-04 2.39734578e-04 -1.04169818e-03
2.24474192e-05 -5.25432153e-04 -1.22918248e-04 -8.76317511e-04
4.11939306e-05 -1.26189524e-03 5.99588336e-04 -1.48048219e-03
1.99408453e-04 -2.65891425e-03 -2.76223861e-04 -3.51271395e-03
-2.98541350e-04 -5.35249966e+11 3.49027952e+09 -5.35249966e+11
-3.49027952e+09]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 39.0),), n_basis=41, period=39.0),
coefficients=[[ 4.56281132e-01 -2.28654312e-01 -3.10013255e-02 -1.21268260e-01
-1.83505407e-02 -5.65697775e-02 -1.62209623e-02 -4.86674005e-02
1.98273606e-02 -6.83374288e-02 2.17717735e-02 -7.10658481e-02
-1.40991503e-02 -3.87624973e-02 2.41335076e-02 5.57329302e-03
1.79919430e-02 -4.14817079e-02 -4.50340792e-03 -5.25428976e-02
7.16903433e-03 -9.30632618e-02 1.64897482e-02 -5.17664524e-02
4.53991920e-03 -2.13261194e-02 4.88771240e-03 -2.89069272e-02
1.38286057e-03 -2.75282139e-02 2.92747171e-02 -3.53459195e-02
-2.63050936e-02 -1.46499558e-01 1.93183125e-02 -1.58244085e-01
-6.11090286e-04 -2.31636813e+13 1.51046665e+11 -2.31636813e+13
-1.51046665e+11]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 39.0),), n_basis=41, period=39.0),
coefficients=[[ 1.49766956e-02 -6.83759827e-03 -7.46119109e-04 -2.33955183e-03
-8.93288487e-04 -1.51897290e-03 2.45595331e-04 -1.39052098e-03
2.24256201e-04 -1.32369870e-03 3.67304047e-05 -1.08348054e-03
-1.03580715e-04 -1.06465385e-03 -4.57530968e-04 -1.03849871e-03
-2.36386943e-04 -7.41662971e-04 1.67721445e-04 -1.14448596e-03
2.46322797e-04 -8.47967295e-04 1.71600755e-04 -9.12621369e-04
-2.25978147e-04 -9.88994648e-04 -2.80889441e-04 -1.11575912e-03
5.48189047e-05 -1.46691143e-03 4.15429950e-04 -2.25688523e-03
5.84344337e-04 -2.81877744e-03 -3.60090499e-04 -4.38392466e-03
-4.00711227e-04 -6.72600647e+11 4.38592137e+09 -6.72600647e+11
-4.38592137e+09]])
print("System 1:")
A1_sample_window_funct_reg = Function_regression(A1_sample_window_combine_balanced,25,["AgeOfCardInDaysAtTimeOfTest"])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
A2_sample_window_funct_reg = Function_regression(A2_sample_window_combine_balanced,25,["AgeOfCardInDaysAtTimeOfTest"])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 24.0),), n_basis=25, period=24.0),
coefficients=[[-3.30251381e-01 5.77845022e-02 6.85925460e-02 7.53361030e-02
-8.59077863e-03 -7.99524601e-02 -4.25451458e-02 2.04466431e-02
-6.90135206e-02 -1.96008773e-02 -2.07318834e-01 2.79586338e-01
-1.21103259e-01 1.47670656e-02 2.69751978e-02 1.30095756e-01
-3.19253142e-02 6.09050243e-02 5.86177481e-02 1.28638396e-01
8.17866550e-02 1.99671319e-01 1.01443086e-01 1.24955568e+14
2.38210629e-01]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 24.0),), n_basis=25, period=24.0),
coefficients=[[ 3.68941776e-03 -9.29468770e-04 -7.31990464e-04 -7.16818352e-04
1.23365060e-04 1.00452861e-03 5.67626196e-04 -1.70666970e-04
7.34440197e-04 2.86643216e-04 2.82510143e-03 -3.97471396e-03
1.44160994e-03 -3.07993101e-04 -6.16213912e-04 -1.92653680e-03
4.58439683e-04 -1.08444363e-03 -5.48884351e-04 -1.88408446e-03
-1.04380082e-03 -2.50343051e-03 -1.34273635e-03 -1.77162578e+12
-3.36652826e-03]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 24.0),), n_basis=25, period=24.0),
coefficients=[[ 1.92731875e-01 1.29792152e-02 -1.57398989e-01 3.66304557e-02
-1.33104842e-02 3.15211241e-02 1.88453651e-02 9.79435374e-02
1.36036849e-02 -2.79887400e-02 1.21688496e-01 -1.41447867e-01
2.10463718e-02 -1.66355295e-03 -1.53974547e-01 -4.45022735e-02
-4.48405611e-02 -7.76704837e-02 -2.33635703e-02 -1.36971813e-01
6.57890479e-03 -3.30767633e-02 2.25921172e-02 -8.34481340e+13
-1.31742526e-01]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 24.0),), n_basis=25, period=24.0),
coefficients=[[ 5.42202812e-03 -1.95077763e-03 -4.88769000e-04 -1.15721474e-03
1.86615137e-04 1.47290831e-03 7.23551211e-04 -6.91710498e-04
1.00267149e-03 1.00280657e-03 4.11294833e-03 -6.61446835e-03
1.73443609e-03 -9.09406654e-04 -6.49465677e-04 -3.52937131e-03
1.21575254e-03 -1.93482202e-03 -1.32681583e-04 -2.60582555e-03
-2.02847562e-03 -4.01651445e-03 -2.42315471e-03 -2.70733084e+12
-5.24267374e-03]])
print("System 1:")
B1_cal_window_funct_reg = Function_regression(B1_cal_window_combine_balanced,90,["AgeOfCardInDaysAtTimeOfTest"])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
B2_cal_window_funct_reg = Function_regression(B2_cal_window_combine_balanced,90,["AgeOfCardInDaysAtTimeOfTest"])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 89.0),), n_basis=91, period=89.0),
coefficients=[[ 1.42891567e+01 -6.10250312e+00 -1.31091754e+00 -2.23531832e+00
-2.52117175e-01 -2.03624564e+00 5.12713196e-01 -1.40187966e+00
-3.62936790e-01 -4.55524459e-01 -4.37531383e-01 -4.92132595e-01
1.18672926e+00 -1.12489975e+00 7.23609537e-01 -8.95270687e-01
6.64170490e-02 -5.59253783e-01 4.91312543e-01 -9.03168009e-01
9.99922269e-01 -1.23226178e+00 5.09369257e-01 -1.45254816e+00
-1.40943791e-02 -1.75619383e+00 1.02597579e-01 -1.43869540e+00
-3.93576611e-01 -1.04289597e+00 -5.94337581e-01 -8.86137689e-01
-7.90927658e-01 -9.00987927e-01 -3.36911177e-01 -1.23181234e+00
-1.05540861e+00 -5.60163906e-02 -1.64872522e+00 5.23194601e-01
-4.71238195e-01 -6.06474053e-01 -1.53959498e-01 -4.16715972e-01
-8.02194913e-01 -2.94407843e-01 -5.42491351e-01 -1.64337603e-01
-6.99831799e-01 4.15880986e-01 -7.16166818e-01 7.82418320e-01
-7.23215989e-01 5.44624787e-01 7.77472114e-03 4.48548385e-01
3.03789184e-01 1.27960234e-01 4.64874049e-01 7.75614698e-02
1.90725386e-01 -4.48736506e-01 3.02154014e-01 -1.62996259e-01
6.27107250e-01 -2.43549457e-01 5.22082159e-01 -3.27734753e-01
1.29237420e-01 -7.33699736e-01 -7.33419139e-02 -6.74324375e-01
3.64857320e-01 -7.32074407e-01 3.90484610e-01 -9.11266957e-01
-4.08465053e-02 -1.63153629e+00 -3.11475024e-01 -1.24712873e+00
-5.94053010e-01 -1.71566588e-01 -4.67250050e-01 -1.52273562e+00
-4.57513183e-01 -2.68000946e+00 8.52869302e-02 -4.88396171e+14
-5.71707857e+12 -4.88396171e+14 5.71707857e+12]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 89.0),), n_basis=91, period=89.0),
coefficients=[[ 3.18257988e-03 -4.22352023e-04 4.81404548e-05 -4.95732917e-04
4.51343312e-04 -9.37262944e-04 -7.23508799e-05 -2.49308802e-04
-8.49124749e-04 -1.95941189e-04 -4.38671757e-04 -3.63405293e-04
1.51774481e-04 7.02784978e-04 -7.98775140e-05 2.46403802e-04
-2.86216768e-04 -1.17456152e-03 5.35227778e-04 -1.24078451e-03
-8.20745071e-05 -1.28758082e-03 2.09037905e-04 9.44049863e-04
6.35770453e-04 -2.51161223e-04 5.40403901e-05 1.23187126e-05
-5.77369599e-04 -4.59417680e-04 3.89668542e-04 -7.50908573e-04
-8.82211079e-04 3.38637120e-04 -7.56532095e-04 4.59012753e-04
-2.77829795e-04 -1.74168672e-04 3.04873857e-04 -3.59617994e-04
8.86485121e-04 4.71313788e-04 -3.32912069e-04 -4.66183323e-04
-8.46617574e-05 1.24765052e-03 8.45550275e-05 2.85953366e-04
-1.50496080e-05 -9.27415403e-04 1.32969456e-04 2.66332929e-04
3.77690247e-05 -9.59645926e-05 2.39662235e-04 -3.60471226e-05
-7.33330659e-05 1.49758847e-03 -8.14519223e-06 -1.14839121e-04
9.36051647e-04 2.92510691e-04 4.45418394e-04 1.13337261e-04
-6.40725844e-05 -5.01127912e-04 -1.52164215e-04 3.55051125e-04
2.36218269e-04 -3.29767680e-04 1.50554166e-03 -1.50101739e-04
-8.20281317e-04 3.71976774e-04 4.48489535e-05 -1.72384177e-04
1.81176389e-04 -5.17208586e-05 -6.58767973e-04 1.51501964e-04
-1.89547357e-04 -5.62940562e-04 2.42909969e-04 -4.21444997e-04
7.19896983e-04 1.39672083e-03 2.58634292e-04 -9.07495900e+10
-1.06229853e+09 -9.07495900e+10 1.06229853e+09]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 89.0),), n_basis=91, period=89.0),
coefficients=[[ 1.72679636e+01 -7.07995356e+00 -1.59143028e+00 -2.76618512e+00
-2.86366913e-01 -2.45979073e+00 5.77075583e-01 -1.68773378e+00
-5.15599420e-01 -6.10387568e-01 -5.86296702e-01 -7.17829270e-01
1.36744302e+00 -1.21655692e+00 8.17817810e-01 -1.06181141e+00
2.95050144e-01 -6.50642569e-01 6.39614935e-01 -1.07211669e+00
1.10408060e+00 -2.17117110e+00 7.31721442e-01 -1.44540895e+00
-1.80027300e-02 -1.98276601e+00 2.10905600e-01 -1.82600597e+00
-5.16858644e-01 -1.22390682e+00 -7.06246345e-01 -1.19744261e+00
-1.11589742e+00 -1.06441503e+00 -4.14105128e-01 -1.53627876e+00
-1.33032022e+00 -1.59316346e-01 -1.91672099e+00 5.50823578e-01
-3.20285066e-01 -7.51217924e-01 -4.12670018e-01 -4.73297612e-01
-7.78473587e-01 -3.96988434e-01 -5.96030583e-01 -1.55754340e-01
-9.04284839e-01 2.58145716e-01 -8.42671808e-01 9.03390587e-01
-7.19861887e-01 7.17062940e-01 2.73575892e-02 5.33109027e-01
2.23131592e-01 1.92997433e-01 4.67950088e-01 -1.93501659e-03
2.92448379e-01 -4.50692030e-01 4.41823620e-01 -1.41422172e-01
8.11244996e-01 -2.22358666e-01 6.12318308e-01 -3.55254932e-01
2.11071729e-01 -8.95543366e-01 2.14474784e-01 -7.26830961e-01
2.42616125e-01 -1.10294662e+00 4.69055773e-01 -8.80449977e-01
-6.80598510e-02 -2.11034183e+00 -5.95194862e-01 -1.35217836e+00
-7.34749384e-01 -2.62154350e-01 -5.41800964e-01 -1.89083798e+00
-4.07060075e-01 -3.03598594e+00 1.73612800e-01 -5.88157585e+14
-6.88486791e+12 -5.88157585e+14 6.88486791e+12]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 89.0),), n_basis=91, period=89.0),
coefficients=[[-4.31143377e-05 -1.12150788e-03 2.71736583e-04 7.62766895e-04
3.41454143e-04 -6.61279640e-04 2.37155906e-04 -1.12011535e-04
-2.51673660e-04 -5.26813564e-05 3.38214180e-05 7.71516022e-04
4.96378834e-04 -3.06532657e-05 2.36328538e-04 3.03205722e-05
-2.29348935e-03 -1.35100519e-03 -4.94563823e-05 -1.19647634e-03
2.65005116e-04 3.77413264e-03 -9.16387610e-04 -7.97097310e-04
1.01758315e-03 -6.44086159e-04 -6.07152148e-04 1.15079281e-03
-4.24797981e-04 -6.56112303e-04 4.88251991e-04 7.99228642e-04
4.59452424e-04 2.90626543e-04 -5.88458247e-04 1.10884827e-03
7.27309513e-04 9.74061014e-04 8.24712432e-05 1.42461003e-04
-1.06851095e-03 9.98719309e-04 1.52884072e-03 -6.17745642e-04
-1.02022496e-03 2.27016985e-03 -2.55217017e-04 -1.70246204e-04
6.41116180e-04 7.06212290e-04 1.80440402e-04 4.74133276e-04
-8.54228237e-04 -3.76467950e-04 2.10789311e-04 -1.01770072e-04
7.87588262e-04 1.22669471e-03 5.45402054e-04 8.74710123e-04
5.59758213e-04 -1.78345762e-04 -1.79993970e-04 -1.37118697e-04
-5.96167139e-04 -9.82705985e-04 -2.74478576e-04 2.64289168e-04
9.93329243e-05 8.17582560e-05 -7.55175193e-04 -6.28234332e-04
5.92207570e-04 1.93325559e-03 6.84784190e-05 -1.34831669e-03
3.63956598e-04 1.73578488e-03 6.99347736e-04 -6.09696523e-04
-2.15981067e-05 1.03016129e-05 2.63823833e-04 3.12103752e-04
-3.16285158e-04 9.47698251e-04 -2.86127734e-04 3.33629758e+09
3.90541051e+07 3.33629758e+09 -3.90541051e+07]])
print("System 1:")
B1_sample_window_funct_reg = Function_regression(B1_sample_window_combine_balanced,20,["AgeOfCardInDaysAtTimeOfTest"])
print("----------------------------------------------------------------------------")
print("\n","System 2:")
B2_sample_window_funct_reg = Function_regression(B2_sample_window_combine_balanced,20,["AgeOfCardInDaysAtTimeOfTest"])
System 1:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 19.0),), n_basis=21, period=19.0),
coefficients=[[ 1.84538430e+00 -6.46372536e-01 2.91777932e-01 -2.57649917e-01
2.60196803e-01 -3.11522012e-01 5.18497112e-01 -1.82107966e-01
3.80027094e-01 -3.13362070e-01 4.52342475e-01 -1.67582580e-01
3.40671750e-01 -3.81452513e-01 1.96463381e-01 -5.73908578e-01
3.90192634e-01 -4.66205501e+14 -1.13431995e+13 -4.66205501e+14
1.13431995e+13]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 19.0),), n_basis=21, period=19.0),
coefficients=[[-1.89880792e-04 2.48330544e-04 -1.85525532e-05 -5.32231777e-04
5.61726885e-04 8.30594356e-04 -9.29579737e-04 1.28008728e-04
2.82027076e-05 7.07856029e-04 -6.19713285e-04 -2.44085661e-04
2.46150211e-04 7.76322851e-04 1.42362182e-03 1.93806187e-04
1.20669119e-04 1.26759647e+11 3.08417632e+09 1.26759647e+11
-3.08417632e+09]])
----------------------------------------------------------------------------
System 2:
Model Summary:
Intercept: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 19.0),), n_basis=21, period=19.0),
coefficients=[[ 2.13442579e+00 -8.30049350e-01 1.91552809e-01 -2.50182509e-01
2.96205937e-01 -2.37989613e-01 6.24244392e-01 -4.55737511e-01
5.13517099e-01 -2.40813419e-01 4.83129010e-01 -2.67266721e-01
5.31998673e-01 -1.99662714e-01 4.10364939e-01 -6.56041573e-01
4.72179615e-01 -5.59289859e+14 -1.36080258e+13 -5.59289859e+14
1.36080258e+13]])
Coefficient of AgeOfCardInDaysAtTimeOfTest: FDataBasis(
_basis=FourierBasis(domain_range=((0.0, 19.0),), n_basis=21, period=19.0),
coefficients=[[ 5.90177405e-04 6.46152356e-04 1.38537804e-03 -1.24947712e-03
1.12082776e-03 -1.95369236e-04 -1.30012442e-03 1.62515284e-03
-4.52117678e-04 -3.63481402e-04 3.83963095e-05 -8.68329532e-06
-4.66714546e-04 -1.22401436e-03 4.56914103e-04 9.57779357e-05
5.70050470e-05 7.55458053e+10 1.83809745e+09 7.55458053e+10
-1.83809745e+09]])
As the result show above, the first time point is larger than others. And apart from Sample Window Sensor A (the last two points), the value at the last 4 time stamps are also significantly greater than the rest of the data.
So for the convenience of visualization, we remove these points.
coefficent_visualization(A1_cal_window_funct_reg,A2_cal_window_funct_reg,["AgeOfCardInDaysAtTimeOfTest"],range(1,36),"SensorA Cal window")
coefficent_visualization(A1_sample_window_funct_reg,A2_sample_window_funct_reg,["AgeOfCardInDaysAtTimeOfTest"],range(1,23),"SensorA sample window")
coefficent_visualization(B1_cal_window_funct_reg,B2_cal_window_funct_reg,["AgeOfCardInDaysAtTimeOfTest"],range(1,86),"SensorB Cal window")
coefficent_visualization(B1_sample_window_funct_reg, B2_sample_window_funct_reg, ["AgeOfCardInDaysAtTimeOfTest"], range(1, 16), "SensorB Sample window")